package cn.itcast.crawler;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

// 设置相关HttpClient的参数
// 由于爬取的网站的访问量及网络情况不同，针对不同的网站我们需要设置不同的爬取参数
// 以应对不同的情况，提高数据的爬取效率
// 下面我们只是设置一些基本的参数，例如响应时间等
// 通过Get请求来演示
public class ConfigTest {
    public static void main(String[] args) {
        // 创建连接池
        PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
        // 设置连接参数
        cm.setMaxTotal(100);
        cm.setDefaultMaxPerRoute(10);
        // get
        doGet(cm);
        doGet(cm);
    }

    private static void doGet(PoolingHttpClientConnectionManager cm) {
        CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
        String uri = "http://www.itcast.cn";
        HttpGet httpGet = new HttpGet(uri);
        RequestConfig config = RequestConfig.custom()
                .setConnectTimeout(1000) // 设置连接超时时间
                .setConnectionRequestTimeout(1000) // 设置连接请求超时时间
                .setSocketTimeout(10 * 1000) // 设置套接字超时时间
                .build();
        httpGet.setConfig(config); // 设置Get请求的参数
        CloseableHttpResponse response = null;
        try {
            response = httpClient.execute(httpGet);
            if (response.getStatusLine().getStatusCode() == 200) {
                String content = EntityUtils.toString(response.getEntity(),"utf8");
                System.out.println(content);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            if (response != null) {
                try {
                    response.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        }
    }
}
